#!/bin/sh
#PBS -q normal
#PBS -l nodes=6:ppn=16
#PBS -l walltime=05:00:00

############### Prepare software ################
source /com/extra/python/2.7/load.sh
source /com/extra/ATLAS/3.9.84/load.sh

# setup in scratch
cd /home/asand/tqDist-experiments
cp *.so /scratch/$PBS_JOBID/
cp qdist_bin /scratch/$PBS_JOBID/
cp qdist_gen /scratch/$PBS_JOBID/
cp *.py /scratch/$PBS_JOBID/
cp treeGen /scratch/$PBS_JOBID/
cp commands* /scratch/$PBS_JOBID/
mkdir /scratch/$PBS_JOBID/trees

## Distribute from scratch
NODES=$(cat $PBS_NODEFILE | uniq | grep -v `hostname -s` | sed 's/n/t/g')
for node in $NODES; do (scp -p -r /scratch/$PBS_JOBID/* $node:/scratch/$PBS_JOBID/.); done

## start jobs
## dispatch jobs -S: print statistics on exit, -u start only one task on one node (uni-core), -r commands: run each command in commands in parallel
cd /scratch/$PBS_JOBID
dispatch -S -u -r commands_bin

# copy from scratch
for node in $nodes; do (scp -p -r $node:/scratch/$PBS_JOBID/andreas_stdout_bin /home/asand/tqDist-experiments/stdout_${node}); done
for node in $nodes; do (scp -p -r $node:/scratch/$PBS_JOBID/andreas_stderr_bin /home/asand/tqDist-experiments/stderr_${node}); done
